set maxvar 5500
set more off

**********************************
** ORGANIZE BES
**********************************

// Read in bes.
use "BES2017_W13_Panel_v1.2.dta", clear

// Keep respondents who answered in any of waves 1, 2, 4, 6, and 7.
keep if wave1==1 | wave2==1 | wave4==1 | wave6==1 | wave7==1

// Variables to keep: all placement variables, all certainty variables, age, gender, party ID, id, wave, education
keep EUIntegrationConW1 EUIntegrationLabW1 EUIntegrationLDW1 EUIntegrationUKIPW1 ///
EUIntegrationConW2 EUIntegrationLabW2 EUIntegrationLDW2 EUIntegrationUKIPW2 ///
EUIntegrationConW4 EUIntegrationLabW4 EUIntegrationLDW4 EUIntegrationUKIPW4 ///
EUIntegrationConW6 EUIntegrationLabW6 EUIntegrationLDW6 EUIntegrationUKIPW6 ///
EUIntegrationConW7 EUIntegrationLabW7 EUIntegrationLDW7 EUIntegrationUKIPW7 ///
certaintyEUConW1 certaintyEULabW1 certaintyEULDW1 certaintyEUUKIPW1 ///
certaintyEUConW2 certaintyEULabW2 certaintyEULDW2 certaintyEUUKIPW2 ///
certaintyEUConW4 certaintyEULabW4 certaintyEULDW4 certaintyEUUKIPW4 ///
certaintyEUConW6 certaintyEULabW6 certaintyEULDW6 certaintyEUUKIPW6 ///
certaintyEUConW7 certaintyEULabW7 certaintyEULDW7 certaintyEUUKIPW7 ///
ageW1 ageW7 gender partyIdW1 partyIdW2 partyIdW4 partyIdW6 partyIdW7 ///
id wave1 wave2 wave4 wave6 wave7 educationW1_W6 educationW7

// Create previous party position variables
gen prevEUIntegrationConW2 = EUIntegrationConW1
gen prevEUIntegrationConW4 = EUIntegrationConW2
gen prevEUIntegrationConW6 = EUIntegrationConW4
gen prevEUIntegrationConW7 = EUIntegrationConW6

gen prevEUIntegrationLabW2 = EUIntegrationLabW1
gen prevEUIntegrationLabW4 = EUIntegrationLabW2
gen prevEUIntegrationLabW6 = EUIntegrationLabW4
gen prevEUIntegrationLabW7 = EUIntegrationLabW6

gen prevEUIntegrationLDW2 = EUIntegrationLDW1
gen prevEUIntegrationLDW4 = EUIntegrationLDW2
gen prevEUIntegrationLDW6 = EUIntegrationLDW4
gen prevEUIntegrationLDW7 = EUIntegrationLDW6

gen prevEUIntegrationUKIPW2 = EUIntegrationUKIPW1
gen prevEUIntegrationUKIPW4 = EUIntegrationUKIPW2
gen prevEUIntegrationUKIPW6 = EUIntegrationUKIPW4
gen prevEUIntegrationUKIPW7 = EUIntegrationUKIPW6

// Create previous party position certainty variables.
gen prevCertaintyEUConW2 = certaintyEUConW1
gen prevCertaintyEUConW4 = certaintyEUConW2
gen prevCertaintyEUConW6 = certaintyEUConW4
gen prevCertaintyEUConW7 = certaintyEUConW6

gen prevCertaintyEULabW2 = certaintyEULabW1
gen prevCertaintyEULabW4 = certaintyEULabW2
gen prevCertaintyEULabW6 = certaintyEULabW4
gen prevCertaintyEULabW7 = certaintyEULabW6

gen prevCertaintyEULDW2 = certaintyEULDW1
gen prevCertaintyEULDW4 = certaintyEULDW2
gen prevCertaintyEULDW6 = certaintyEULDW4
gen prevCertaintyEULDW7 = certaintyEULDW6

gen prevCertaintyEUUKIPW2 = certaintyEUUKIPW1
gen prevCertaintyEUUKIPW4 = certaintyEUUKIPW2
gen prevCertaintyEUUKIPW6 = certaintyEUUKIPW4
gen prevCertaintyEUUKIPW7 = certaintyEUUKIPW6

// Clean up placement variables:
replace EUIntegrationConW1 = . if EUIntegrationConW1 == 9999
replace EUIntegrationConW2 = . if EUIntegrationConW2 == 9999
replace EUIntegrationConW4 = . if EUIntegrationConW4 == 9999
replace EUIntegrationConW6 = . if EUIntegrationConW6 == 9999
replace EUIntegrationConW7 = . if EUIntegrationConW7 == 9999

replace EUIntegrationLabW1 = . if EUIntegrationLabW1 == 9999
replace EUIntegrationLabW2 = . if EUIntegrationLabW2 == 9999
replace EUIntegrationLabW4 = . if EUIntegrationLabW4 == 9999
replace EUIntegrationLabW6 = . if EUIntegrationLabW6 == 9999
replace EUIntegrationLabW7 = . if EUIntegrationLabW7 == 9999

replace EUIntegrationLDW1 = . if EUIntegrationLDW1 == 9999
replace EUIntegrationLDW2 = . if EUIntegrationLDW2 == 9999
replace EUIntegrationLDW4 = . if EUIntegrationLDW4 == 9999
replace EUIntegrationLDW6 = . if EUIntegrationLDW6 == 9999
replace EUIntegrationLDW7 = . if EUIntegrationLDW7 == 9999

replace EUIntegrationUKIPW1 = . if EUIntegrationUKIPW1 == 9999
replace EUIntegrationUKIPW2 = . if EUIntegrationUKIPW2 == 9999
replace EUIntegrationUKIPW4 = . if EUIntegrationUKIPW4 == 9999
replace EUIntegrationUKIPW6 = . if EUIntegrationUKIPW6 == 9999
replace EUIntegrationUKIPW7 = . if EUIntegrationUKIPW7 == 9999

// Reshape the data to long format. 
reshape long EUIntegration certaintyEU prevEUIntegration prevCertaintyEU, i(id) string

// Create separate variables for the party being asked about and the wave in which it was asked.
generate party = regexs(0) if regexm(_j, "^[^\W]*")
generate wave = regexs(0) if regexm(_j, "[0-9]*$")

// Drop the string variable. 
drop _j 

// Change wave to numeric
destring wave, replace

// Empty out partyId answers if it's not an answer relevant to the wave.
replace partyIdW1 = . if wave !=1
replace partyIdW2 = . if wave !=2
replace partyIdW4 = . if wave !=4
replace partyIdW6 = . if wave !=6
replace partyIdW7 = . if wave !=7

// Create a partyId variable by duplicating partyIdW1.
clonevar partyId = partyIdW1

// Assign the values of partyIdW2~7 to partyId. 
replace partyId = partyIdW2 if !missing(partyIdW2)
replace partyId = partyIdW4 if !missing(partyIdW4)
replace partyId = partyIdW6 if !missing(partyIdW6)
replace partyId = partyIdW7 if !missing(partyIdW7)

// Drop the separate partyIdW1~7 variables.
drop partyIdW1 partyIdW2 partyIdW4 partyIdW6 partyIdW7

// Assign ageW7 to observations with missing responses in ageW1. These are people who did not participate in wave 1.
replace ageW1 = ageW7 if missing(ageW1)
// Rename ageW1 to age.
rename ageW1 age
// Delete ageW7.
drop ageW7

// Check missing values on educationW1_W6.
tabulate educationW1_W6
// Add values of educationW7 to missing observations in educationW1_W6.
replace educationW1_W6 = educationW7 if missing(educationW1_W6)
tabulate educationW1_W6 
rename educationW1_W6 education
drop educationW7

// The wave variable indicates the wave in which a particular question was asked. It does not mean that the respondent actually participated in the wave.
tabulate wave // There are 5 observations for each respondent, regardless of whether the respondent participated in the wave.
// Create answered variable so that it indicates the value (0 or 1) under the correct wave number.
gen answered = wave
replace answered = wave1 if answered == 1
replace answered = wave2 if answered == 2
replace answered = wave4 if answered == 4
replace answered = wave6 if answered == 6
replace answered = wave7 if answered == 7
// Delete observations with 0 in wave.
drop if answered == 0
tabulate answered

// Drop unnecessary variables
drop wave1 wave2 wave4 wave6 wave7 answered

// Change variable names for clarity
rename EUIntegration partyPosition
rename certaintyEU partyPositionCertainty
rename prevEUIntegration prevPartyPosition
rename prevCertaintyEU prevPartyPositionCertainty

// Create party_wave variable.
label define wave 1 "1" 2 "2" 4 "4" 6 "6" 7 "7"
label values wave wave
decode wave, generate(wave_num)
gen partyname_wave = party + wave_num
drop wave_num

// Save the data.
save Data_bes_subset.dta, replace

**********************************
** ORGANIZE CCD
**********************************

// Read in excel file.
import excel "party_newspaper_data_allwaves.xlsx", firstrow clear

// Calculate sum of share_Immigration_positive and share_EU_positive. They indicate leftist positions.
gen share_immigEU_positive = share_Immigration_positive + share_EU_positive

// Calculate sum of share_Immigration_negative and share_EU_negative. They indicate rightist positions.
gen share_immigEU_negative = share_Immigration_negative + share_EU_negative

// Assign the party names used in the bes data to this data.
replace party = "LD" if party == "Liberal Dem"
replace party = "Lab" if party == "Labour"
replace party = "Con" if party == "Conservatives"

// Make a variable combining partyname and wave
label define bes_wave 1 "1" 2 "2" 4 "4" 6 "6" 7 "7"
label values bes_wave bes_wave
decode bes_wave, generate(bes_wave_num)
gen partyname_wave = partyname + bes_wave_num

// Create a variable subtracting share_immigEU_positive from share_immigEU_negative
gen cmpLikeImmigEUPosition = share_immigEU_negative - share_immigEU_positive

// Create a variable subtracting share_EU_positive from share_EU_negative
gen cmpLikeEUPosition = share_EU_negative - share_EU_positive

// Sort the data by party-wave.
sort partyname_wave

// Create lagged cmpLikeEUPosition and cmpLikeImmigEUPosition
sort partyname bes_wave_num
by partyname: gen prevCmpLikeEUPosition = cmpLikeEUPosition[_n-1]
by partyname: gen prevCmpLikeImmigEUPosition = cmpLikeImmigEUPosition[_n-1]

// Drop SNP data
drop if partyname=="SNP"

// Save the data
save Data_party_subset.dta, replace

**********************************
** COMBINE BES AND CCD
**********************************

// Merge party subset to bes subset
use "Data_bes_subset.dta"
tabulate partyname_wave

merge m:1 partyname_wave using Data_party_subset.dta

// Get rid of unnecessary variables.
drop partyname_wave bes_wave_num country

// Sort id and wave
sort id wave

// Rearrange the order of columns.
order id wave party

// Drop _merge
drop _merge

// Change partyId variable to string
decode partyId, gen(partyIdString)
drop partyId
rename partyIdString partyId

// Change the values of partyId to the labeling in party
replace partyId = "Con" if partyId == "Conservative"
replace partyId = "Lab" if partyId == "Labour"
replace partyId = "LD" if partyId == "Liberal Democrat"
replace partyId = "UKIP" if partyId == "United Kingdom Independence Party (UKIP)"

// Create an indicator of copartisanship
gen copartisan = .
replace copartisan = 1 if partyId == party
replace copartisan = 0 if partyId != party

// Convert gender variable to male variable
decode gender, gen(genderString)
drop gender
gen male = .
replace male = 1 if genderString == "Male"
replace male = 0 if genderString == "Female"

// Create squared age variable
gen age_sq = (age)^2

// Create numeric education variable
decode education, gen(education_string)
drop education
rename education_string education
gen education_numeric = .
replace education_numeric = 2 if education == "No formal qualifications"
replace education_numeric = 3 if education == "Youth training certificate/skillseekers"
replace education_numeric = 3 if education == "Recognised trade apprenticeship completed"
replace education_numeric = 3 if education == "Clerical and commercial"
replace education_numeric = 3 if education == "City and Guild certificate"
replace education_numeric = 4 if education == "City and Guild certificate - advanced"
replace education_numeric = 4 if education == "onc"
replace education_numeric = 3 if education == "CSE grades 2-5"
replace education_numeric = 4 if education == "CSE grade 1, GCE O level, GCSE, School Certificate"
replace education_numeric = 4 if education == "Scottish Ordinary/ Lower Certificate"
replace education_numeric = 5 if education == "GCE A level or Higher Certificate"
replace education_numeric = 5 if education == "Scottish Higher Certificate"
replace education_numeric = 6 if education == "Nursing qualification (eg SEN, SRN, SCM, RGN)"
replace education_numeric = 6 if education == "Teaching qualification (not degree)"
replace education_numeric = 6 if education == "University diploma"
replace education_numeric = 7 if education == "University or CNAA first degree (eg BA, B.Sc, B.Ed)"
replace education_numeric = 8 if education == "University or CNAA higher degree (eg M.Sc, Ph.D)"
replace education_numeric = 8 if education == "Other technical, professional or higher qualification"

// Convert Don't know values in partyPosition and prevPartyPosition to missing.
tabulate partyPosition
inspect partyPosition
replace partyPosition = . if partyPosition == 9999

tabulate prevPartyPosition
replace prevPartyPosition = . if prevPartyPosition == 9999

// Create different variables for certainty that code don't know values as maximally uncertain (0)
generate certaintyWithDK2 = partyPositionCertainty
replace certaintyWithDK2 = 0 if certaintyWithDK2 == 9999
generate prevCertaintyWithDK2 = prevPartyPositionCertainty
replace prevCertaintyWithDK2 = 0 if prevCertaintyWithDK2 == 9999

// Convert Don't know values in partyPositionCertainty and prevPartyPositionCertainty to missing.
tabulate partyPositionCertainty
inspect partyPositionCertainty
replace partyPositionCertainty = . if partyPositionCertainty == 9999
tab partyPositionCertainty, nolab

tabulate prevPartyPositionCertainty
replace prevPartyPositionCertainty = . if prevPartyPositionCertainty == 9999

// Create a variable indicating absolute change in eu positions
gen absChangeInEuPosit = abs(cmpLikeEUPosition - prevCmpLikeEUPosition)
gen absChangeInImmigEuPosit = abs(cmpLikeImmigEUPosition - prevCmpLikeImmigEUPosition)

// Create a variable indicating directional change in eu positions
gen ChangeInEuPosit = cmpLikeEUPosition - prevCmpLikeEUPosition
gen ChangeInImmigEuPosit = cmpLikeImmigEUPosition - prevCmpLikeImmigEUPosition

// Duplicate partyId variable
clonevar partyIdFirst = partyId

// Put in wave 1's partyId value for each id in partyIdFirst
by id: replace partyIdFirst = partyIdFirst[1]

// Create new copartisan variable
generate copartisanFirst = 1 if party == partyIdFirst
replace copartisanFirst = 0 if party!=partyIdFirst

// Create measures of perception change
sort id party wave
gen perc_ch = partyPosition-partyPosition[_n-1] if id==id[_n-1] & party==party[_n-1]

// Keep only variables we will use
keep perc_ch partyPositionCertainty ChangeInImmigEuPosit absChangeInImmigEuPosit copartisan age male education_numeric wave party id bes_wave ChangeInEuPosit copartisanFirst prevPartyPositionCertainty absChangeInEuPosit certaintyWithDK2 partyname

// Save the data
save "Data_besparty.dta", replace
